{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python2.7/dist-packages/matplotlib/font_manager.py:273: UserWarning: Matplotlib is building the font cache using fc-list. This may take a moment.\n",
      "  warnings.warn('Matplotlib is building the font cache using fc-list. This may take a moment.')\n"
     ]
    }
   ],
   "source": [
    "# dependencies\n",
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "from sklearn.model_selection import train_test_split\n",
    "import time\n",
    "import data_utils\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# read dataset\n",
    "X, Y, en_word2idx, en_idx2word, en_vocab, de_word2idx, de_idx2word, de_vocab = data_utils.read_dataset('data.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sentence in English - encoded: [108, 5, 867, 93, 38, 25, 2583]\n",
      "Sentence in German - encoded: [166, 262, 8, 474, 268, 324, 67, 15, 130]\n",
      "Decoded:\n",
      "------------------------\n",
      "They walk in here and \n",
      "\n",
      "Die kommen hier herein und\n"
     ]
    }
   ],
   "source": [
    "# inspect data\n",
    "print 'Sentence in English - encoded:', X[0]\n",
    "print 'Sentence in German - encoded:', Y[0]\n",
    "print 'Decoded:\\n------------------------'\n",
    "\n",
    "for i in range(len(X[1])):\n",
    "    print en_idx2word[X[1][i]],\n",
    "    \n",
    "print '\\n'\n",
    "\n",
    "for i in range(len(Y[1])):\n",
    "    print de_idx2word[Y[1][i]],"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# data processing\n",
    "\n",
    "# data padding\n",
    "def data_padding(x, y, length = 15):\n",
    "    for i in range(len(x)):\n",
    "        x[i] = x[i] + (length - len(x[i])) * [en_word2idx['<pad>']]\n",
    "        y[i] = [de_word2idx['<go>']] + y[i] + [de_word2idx['<eos>']] + (length-len(y[i])) * [de_word2idx['<pad>']]\n",
    "\n",
    "data_padding(X, Y)\n",
    "\n",
    "# data splitting\n",
    "X_train,  X_test, Y_train, Y_test = train_test_split(X, Y, test_size = 0.1)\n",
    "\n",
    "del X\n",
    "del Y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# build a model\n",
    "\n",
    "input_seq_len = 15\n",
    "output_seq_len = 17\n",
    "en_vocab_size = len(en_vocab) + 2 # + <pad>, <ukn>\n",
    "de_vocab_size = len(de_vocab) + 4 # + <pad>, <ukn>, <eos>, <go>\n",
    "\n",
    "# placeholders\n",
    "encoder_inputs = [tf.placeholder(dtype = tf.int32, shape = [None], name = 'encoder{}'.format(i)) for i in range(input_seq_len)]\n",
    "decoder_inputs = [tf.placeholder(dtype = tf.int32, shape = [None], name = 'decoder{}'.format(i)) for i in range(output_seq_len)]\n",
    "\n",
    "targets = [decoder_inputs[i+1] for i in range(output_seq_len-1)]\n",
    "# add one more target\n",
    "targets.append(tf.placeholder(dtype = tf.int32, shape = [None], name = 'last_target'))\n",
    "target_weights = [tf.placeholder(dtype = tf.float32, shape = [None], name = 'target_w{}'.format(i)) for i in range(output_seq_len)]\n",
    "\n",
    "# output projection\n",
    "size = 512\n",
    "w_t = tf.get_variable('proj_w', [de_vocab_size, size], tf.float32)\n",
    "b = tf.get_variable('proj_b', [de_vocab_size], tf.float32)\n",
    "w = tf.transpose(w_t)\n",
    "output_projection = (w, b)\n",
    "\n",
    "outputs, states = tf.contrib.legacy_seq2seq.embedding_attention_seq2seq(\n",
    "                                            encoder_inputs,\n",
    "                                            decoder_inputs,\n",
    "                                            tf.contrib.rnn.BasicLSTMCell(size),\n",
    "                                            num_encoder_symbols = en_vocab_size,\n",
    "                                            num_decoder_symbols = de_vocab_size,\n",
    "                                            embedding_size = 100,\n",
    "                                            feed_previous = False,\n",
    "                                            output_projection = output_projection,\n",
    "                                            dtype = tf.float32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# define our loss function\n",
    "\n",
    "# sampled softmax loss - returns: A batch_size 1-D tensor of per-example sampled softmax losses\n",
    "def sampled_loss(labels, logits):\n",
    "    return tf.nn.sampled_softmax_loss(\n",
    "                        weights = w_t,\n",
    "                        biases = b,\n",
    "                        labels = tf.reshape(labels, [-1, 1]),\n",
    "                        inputs = logits,\n",
    "                        num_sampled = 512,\n",
    "                        num_classes = de_vocab_size)\n",
    "\n",
    "# Weighted cross-entropy loss for a sequence of logits\n",
    "loss = tf.contrib.legacy_seq2seq.sequence_loss(outputs, targets, target_weights, softmax_loss_function = sampled_loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# let's define some helper functions\n",
    "\n",
    "# simple softmax function\n",
    "def softmax(x):\n",
    "    n = np.max(x)\n",
    "    e_x = np.exp(x - n)\n",
    "    return e_x / e_x.sum()\n",
    "\n",
    "# feed data into placeholders\n",
    "def feed_dict(x, y, batch_size = 64):\n",
    "    feed = {}\n",
    "    \n",
    "    idxes = np.random.choice(len(x), size = batch_size, replace = False)\n",
    "    \n",
    "    for i in range(input_seq_len):\n",
    "        feed[encoder_inputs[i].name] = np.array([x[j][i] for j in idxes], dtype = np.int32)\n",
    "        \n",
    "    for i in range(output_seq_len):\n",
    "        feed[decoder_inputs[i].name] = np.array([y[j][i] for j in idxes], dtype = np.int32)\n",
    "        \n",
    "    feed[targets[len(targets)-1].name] = np.full(shape = [batch_size], fill_value = de_word2idx['<pad>'], dtype = np.int32)\n",
    "    \n",
    "    for i in range(output_seq_len-1):\n",
    "        batch_weights = np.ones(batch_size, dtype = np.float32)\n",
    "        target = feed[decoder_inputs[i+1].name]\n",
    "        for j in range(batch_size):\n",
    "            if target[j] == de_word2idx['<pad>']:\n",
    "                batch_weights[j] = 0.0\n",
    "        feed[target_weights[i].name] = batch_weights\n",
    "        \n",
    "    feed[target_weights[output_seq_len-1].name] = np.zeros(batch_size, dtype = np.float32)\n",
    "    \n",
    "    return feed\n",
    "\n",
    "# decode output sequence\n",
    "def decode_output(output_seq):\n",
    "    words = []\n",
    "    for i in range(output_seq_len):\n",
    "        smax = softmax(output_seq[i])\n",
    "        idx = np.argmax(smax)\n",
    "        words.append(de_idx2word[idx])\n",
    "    return words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# ops and hyperparameters\n",
    "learning_rate = 5e-3\n",
    "batch_size = 64\n",
    "steps = 1000\n",
    "\n",
    "# ops for projecting outputs\n",
    "outputs_proj = [tf.matmul(outputs[i], output_projection[0]) + output_projection[1] for i in range(output_seq_len)]\n",
    "\n",
    "# training op\n",
    "optimizer = tf.train.RMSPropOptimizer(learning_rate).minimize(loss)\n",
    "\n",
    "# init op\n",
    "init = tf.global_variables_initializer()\n",
    "\n",
    "# forward step\n",
    "def forward_step(sess, feed):\n",
    "    output_sequences = sess.run(outputs_proj, feed_dict = feed)\n",
    "    return output_sequences\n",
    "\n",
    "# training step\n",
    "def backward_step(sess, feed):\n",
    "    sess.run(optimizer, feed_dict = feed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "------------------TRAINING------------------\n",
      "step: 0, loss: 9.15606880188\n",
      "step: 4, loss: 9.13638210297\n",
      "step: 9, loss: 9.06076049805\n",
      "step: 14, loss: 9.14947128296\n",
      "step: 19, loss: 9.05413341522\n",
      "Checkpoint is saved\n",
      "step: 24, loss: 8.86143398285\n",
      "step: 29, loss: 8.98340415955\n",
      "step: 34, loss: 8.61945915222\n",
      "step: 39, loss: 8.50632095337\n",
      "Checkpoint is saved\n",
      "step: 44, loss: 7.5141119957\n",
      "step: 49, loss: 6.97522449493\n",
      "step: 54, loss: 7.09013032913\n",
      "step: 59, loss: 6.67141437531\n",
      "Checkpoint is saved\n",
      "step: 64, loss: 6.38296127319\n",
      "step: 69, loss: 6.71281719208\n",
      "step: 74, loss: 6.34897422791\n",
      "step: 79, loss: 5.55081653595\n",
      "Checkpoint is saved\n",
      "step: 84, loss: 6.7832198143\n",
      "step: 89, loss: 5.41764545441\n",
      "step: 94, loss: 5.77264690399\n",
      "step: 99, loss: 5.74886131287\n",
      "Checkpoint is saved\n",
      "step: 104, loss: 5.41649436951\n",
      "step: 109, loss: 5.76595020294\n",
      "step: 114, loss: 5.90559673309\n",
      "step: 119, loss: 5.5111284256\n",
      "Checkpoint is saved\n",
      "step: 124, loss: 4.90815448761\n",
      "step: 129, loss: 5.23872709274\n",
      "step: 134, loss: 4.89144515991\n",
      "step: 139, loss: 5.05637788773\n",
      "Checkpoint is saved\n",
      "step: 144, loss: 5.13223648071\n",
      "step: 149, loss: 5.60739707947\n",
      "step: 154, loss: 4.76036691666\n",
      "step: 159, loss: 4.89955949783\n",
      "Checkpoint is saved\n",
      "step: 164, loss: 4.76937866211\n",
      "step: 169, loss: 5.71598529816\n",
      "step: 174, loss: 4.89542913437\n",
      "step: 179, loss: 5.59620094299\n",
      "Checkpoint is saved\n",
      "step: 184, loss: 4.44553899765\n",
      "step: 189, loss: 5.2160859108\n",
      "step: 194, loss: 4.46093845367\n",
      "step: 199, loss: 4.196164608\n",
      "Checkpoint is saved\n",
      "step: 204, loss: 4.64937782288\n",
      "step: 209, loss: 4.5072889328\n",
      "step: 214, loss: 4.23183584213\n",
      "step: 219, loss: 4.85551452637\n",
      "Checkpoint is saved\n",
      "step: 224, loss: 4.19429922104\n",
      "step: 229, loss: 4.19019937515\n",
      "step: 234, loss: 4.14655590057\n",
      "step: 239, loss: 4.22521686554\n",
      "Checkpoint is saved\n",
      "step: 244, loss: 4.16123247147\n",
      "step: 249, loss: 3.93684101105\n",
      "step: 254, loss: 4.13872051239\n",
      "step: 259, loss: 3.75406932831\n",
      "Checkpoint is saved\n",
      "step: 264, loss: 3.85143232346\n",
      "step: 269, loss: 3.72330379486\n",
      "step: 274, loss: 3.36678910255\n",
      "step: 279, loss: 3.60818243027\n",
      "Checkpoint is saved\n",
      "step: 284, loss: 3.52965974808\n",
      "step: 289, loss: 3.38069462776\n",
      "step: 294, loss: 3.50864267349\n",
      "step: 299, loss: 3.58087944984\n",
      "Checkpoint is saved\n",
      "step: 304, loss: 3.26785993576\n",
      "step: 309, loss: 3.14019966125\n",
      "step: 314, loss: 3.22744417191\n",
      "step: 319, loss: 3.11994695663\n",
      "Checkpoint is saved\n",
      "step: 324, loss: 3.35502338409\n",
      "step: 329, loss: 3.34768939018\n",
      "step: 334, loss: 3.47381138802\n",
      "step: 339, loss: 3.0688829422\n",
      "Checkpoint is saved\n",
      "step: 344, loss: 3.26070594788\n",
      "step: 349, loss: 3.05061817169\n",
      "step: 354, loss: 2.99904680252\n",
      "step: 359, loss: 2.92647647858\n",
      "Checkpoint is saved\n",
      "step: 364, loss: 2.72477197647\n",
      "step: 369, loss: 2.91658115387\n",
      "step: 374, loss: 3.00638914108\n",
      "step: 379, loss: 2.91731452942\n",
      "Checkpoint is saved\n",
      "step: 384, loss: 3.32163047791\n",
      "step: 389, loss: 2.55599832535\n",
      "step: 394, loss: 2.63983058929\n",
      "step: 399, loss: 2.97371339798\n",
      "Checkpoint is saved\n",
      "step: 404, loss: 2.70973396301\n",
      "step: 409, loss: 2.81094098091\n",
      "step: 414, loss: 2.72978258133\n",
      "step: 419, loss: 2.73652839661\n",
      "Checkpoint is saved\n",
      "step: 424, loss: 2.53156208992\n",
      "step: 429, loss: 2.42091226578\n",
      "step: 434, loss: 2.60294556618\n",
      "step: 439, loss: 2.67081975937\n",
      "Checkpoint is saved\n",
      "step: 444, loss: 2.55804896355\n",
      "step: 449, loss: 2.66216111183\n",
      "step: 454, loss: 2.69467425346\n",
      "step: 459, loss: 2.66144800186\n",
      "Checkpoint is saved\n",
      "step: 464, loss: 2.44766807556\n",
      "step: 469, loss: 2.42999339104\n",
      "step: 474, loss: 2.32888698578\n",
      "step: 479, loss: 2.67797231674\n",
      "Checkpoint is saved\n",
      "step: 484, loss: 2.75421738625\n",
      "step: 489, loss: 2.52963280678\n",
      "step: 494, loss: 2.70533132553\n",
      "step: 499, loss: 2.16498565674\n",
      "Checkpoint is saved\n",
      "step: 504, loss: 2.16946458817\n",
      "step: 509, loss: 2.53628182411\n",
      "step: 514, loss: 2.0325551033\n",
      "step: 519, loss: 2.52245783806\n",
      "Checkpoint is saved\n",
      "step: 524, loss: 2.0741276741\n",
      "step: 529, loss: 1.99734139442\n",
      "step: 534, loss: 2.14513015747\n",
      "step: 539, loss: 2.23183107376\n",
      "Checkpoint is saved\n",
      "step: 544, loss: 1.94827413559\n",
      "step: 549, loss: 1.90173244476\n",
      "step: 554, loss: 2.07611751556\n",
      "step: 559, loss: 2.0695271492\n",
      "Checkpoint is saved\n",
      "step: 564, loss: 2.44459724426\n",
      "step: 569, loss: 2.05536794662\n",
      "step: 574, loss: 1.82165920734\n",
      "step: 579, loss: 1.94350862503\n",
      "Checkpoint is saved\n",
      "step: 584, loss: 1.9879963398\n",
      "step: 589, loss: 2.01098108292\n",
      "step: 594, loss: 2.17687869072\n",
      "step: 599, loss: 1.99762034416\n",
      "Checkpoint is saved\n",
      "step: 604, loss: 2.1070432663\n",
      "step: 609, loss: 1.58825445175\n",
      "step: 614, loss: 1.72863912582\n",
      "step: 619, loss: 1.77460610867\n",
      "Checkpoint is saved\n",
      "step: 624, loss: 1.89484548569\n",
      "step: 629, loss: 1.80205273628\n",
      "step: 634, loss: 2.4037270546\n",
      "step: 639, loss: 1.77804780006\n",
      "Checkpoint is saved\n",
      "step: 644, loss: 1.70387148857\n",
      "step: 649, loss: 1.79219937325\n",
      "step: 654, loss: 1.81829106808\n",
      "step: 659, loss: 1.9995470047\n",
      "Checkpoint is saved\n",
      "step: 664, loss: 1.64945363998\n",
      "step: 669, loss: 1.42850506306\n",
      "step: 674, loss: 1.34063410759\n",
      "step: 679, loss: 1.43863761425\n",
      "Checkpoint is saved\n",
      "step: 684, loss: 1.42459189892\n",
      "step: 689, loss: 1.73061084747\n",
      "step: 694, loss: 1.85913681984\n",
      "step: 699, loss: 1.68623590469\n",
      "Checkpoint is saved\n",
      "step: 704, loss: 1.53039968014\n",
      "step: 709, loss: 1.61156439781\n",
      "step: 714, loss: 1.72754395008\n",
      "step: 719, loss: 1.70899713039\n",
      "Checkpoint is saved\n",
      "step: 724, loss: 1.69792103767\n",
      "step: 729, loss: 1.7479197979\n",
      "step: 734, loss: 1.47291517258\n",
      "step: 739, loss: 1.44678342342\n",
      "Checkpoint is saved\n",
      "step: 744, loss: 1.54906415939\n",
      "step: 749, loss: 1.77534091473\n",
      "step: 754, loss: 1.47198927402\n",
      "step: 759, loss: 1.67229151726\n",
      "Checkpoint is saved\n",
      "step: 764, loss: 1.51699376106\n",
      "step: 769, loss: 1.443395257\n",
      "step: 774, loss: 1.68605959415\n",
      "step: 779, loss: 1.80405914783\n",
      "Checkpoint is saved\n",
      "step: 784, loss: 1.40637636185\n",
      "step: 789, loss: 1.34082770348\n",
      "step: 794, loss: 1.43669247627\n",
      "step: 799, loss: 1.34441137314\n",
      "Checkpoint is saved\n",
      "step: 804, loss: 1.26650381088\n",
      "step: 809, loss: 1.68948316574\n",
      "step: 814, loss: 1.49345564842\n",
      "step: 819, loss: 1.29472160339\n",
      "Checkpoint is saved\n",
      "step: 824, loss: 1.44468212128\n",
      "step: 829, loss: 1.17797875404\n",
      "step: 834, loss: 1.35859322548\n",
      "step: 839, loss: 1.32436037064\n",
      "Checkpoint is saved\n",
      "step: 844, loss: 1.29831290245\n",
      "step: 849, loss: 1.22677099705\n",
      "step: 854, loss: 1.13562202454\n",
      "step: 859, loss: 1.42926514149\n",
      "Checkpoint is saved\n",
      "step: 864, loss: 1.12431359291\n",
      "step: 869, loss: 1.32365953922\n",
      "step: 874, loss: 1.23509478569\n",
      "step: 879, loss: 1.71939682961\n",
      "Checkpoint is saved\n",
      "step: 884, loss: 1.36636090279\n",
      "step: 889, loss: 1.06496393681\n",
      "step: 894, loss: 1.28424942493\n",
      "step: 899, loss: 1.3816177845\n",
      "Checkpoint is saved\n",
      "step: 904, loss: 1.48245871067\n",
      "step: 909, loss: 1.30291283131\n",
      "step: 914, loss: 1.31340122223\n",
      "step: 919, loss: 1.14693808556\n",
      "Checkpoint is saved\n",
      "step: 924, loss: 1.03125548363\n",
      "step: 929, loss: 1.31596028805\n",
      "step: 934, loss: 1.02932834625\n",
      "step: 939, loss: 1.46618723869\n",
      "Checkpoint is saved\n",
      "step: 944, loss: 1.25197148323\n",
      "step: 949, loss: 1.32616424561\n",
      "step: 954, loss: 0.872643768787\n",
      "step: 959, loss: 1.04848980904\n",
      "Checkpoint is saved\n",
      "step: 964, loss: 1.13522076607\n",
      "step: 969, loss: 1.01238715649\n",
      "step: 974, loss: 1.26613914967\n",
      "step: 979, loss: 1.56620335579\n",
      "Checkpoint is saved\n",
      "step: 984, loss: 1.16490602493\n",
      "step: 989, loss: 1.52711963654\n",
      "step: 994, loss: 1.25121533871\n",
      "step: 999, loss: 1.07643723488\n",
      "Checkpoint is saved\n",
      "Training time for 1000 steps: 8006.03512812s\n"
     ]
    }
   ],
   "source": [
    "# let's train the model\n",
    "\n",
    "# we will use this list to plot losses through steps\n",
    "losses = []\n",
    "\n",
    "# save a checkpoint so we can restore the model later \n",
    "saver = tf.train.Saver()\n",
    "\n",
    "print '------------------TRAINING------------------'\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    sess.run(init)\n",
    "    \n",
    "    t = time.time()\n",
    "    for step in range(steps):\n",
    "        feed = feed_dict(X_train, Y_train)\n",
    "            \n",
    "        backward_step(sess, feed)\n",
    "        \n",
    "        if step % 5 == 4 or step == 0:\n",
    "            loss_value = sess.run(loss, feed_dict = feed)\n",
    "            print 'step: {}, loss: {}'.format(step, loss_value)\n",
    "            losses.append(loss_value)\n",
    "        \n",
    "        if step % 20 == 19:\n",
    "            saver.save(sess, 'checkpoints/', global_step=step)\n",
    "            print 'Checkpoint is saved'\n",
    "            \n",
    "    print 'Training time for {} steps: {}s'.format(steps, time.time() - t)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAbIAAAEiCAYAAACcFVdfAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xdg1PX9P/Dn5/O5nVvZIYEwQkIYYYU9IlJcOAqOukC0\nX9o6C0q1iKPVuqr9AnVVq/bXVkUtUtt+CyoiCDLCCCABGWGEQAaZl7vL7bvP748LBzEJBEhyucvz\n8Q/k8uHudR+OPHlvwWKxyCAiIopQYrgLICIiuhQMMiIiimgMMiIiimgMMiIiimgMMiIiimgMMiIi\nimidFmQPPvggMjMzMWHChNBjTz/9NMaMGYNJkyZh9uzZsFqtnVUOERFFiU4LsjvvvBMrVqxo8tjU\nqVORn5+PjRs3IiMjA0uWLOmscoiIKEp0WpCNHz8eZrO5yWNTpkyBKAZLGDVqFEpLSzurHCIiihJd\nZozsgw8+wBVXXBHuMoiIKMJ0iSD7wx/+AKVSiVtuuSXcpRARUYRRhLuAZcuW4auvvsJ//vOfcJdC\nREQRqFODTJab7k+8Zs0avPrqq1i1ahXUanVnlkJERFGi07oW586di6uuugqHDx/GkCFD8MEHH+Cx\nxx6D3W7HjBkzkJeXhwULFnRWOdRGRUVF4S6hW+J9Dw/e98jUaS2yd999t9ljs2bN6qyXJyKiKNUl\nJnsQERFdLAYZERFFNAYZERFFNAYZERFFNAYZERFFNAYZERFFNAYZERFFNAYZERFFNAYZERFFNAYZ\nERFFNAYZERFFNAYZERFFNAYZERFFNAYZERFFNAYZERFFNAYZERFFNAYZERFFNAYZERFFNAYZERFF\nNAYZERFFNAYZERFFNAYZERFFNAYZERFFNAYZERFFNAYZERFFNAYZERFFNAYZERFFNAYZERFFNAYZ\nERFFNAYZERFFNAYZERFFtE4LsgcffBCZmZmYMGFC6DGLxYKZM2di1KhRuPHGG1FfX99Z5RARUZTo\ntCC78847sWLFiiaPLVmyBFOmTMGOHTuQl5eHJUuWdFY5REQUJTotyMaPHw+z2dzksVWrVuH2228H\nANx+++1YuXJlZ5VDRERRIqxjZFVVVUhKSgIAJCcno7q6OpzlEBFRBOJkDyIiimiKcL54UlISKisr\nkZSUhFOnTiExMfG8f6aoqKgTKqOz8Z6HB+97ePC+d67MzMxLfo5ODTJZlpt8fc0112DZsmWYP38+\nPvroI0yfPv28z9Eeb5rarqioiPc8DHjfw4P3PTJ1Wtfi3LlzcdVVV+Hw4cMYMmQIPvjgAzz88MNY\nt24dRo0ahW+++Qbz58/vrHKIiChKdFqL7N13323x8X//+9+dVQIREUUhTvYgIqKIxiAjIqKIxiAj\nIqKIxiAjIqKIxiAjIqKIxiAjIqKIxiAjIqKIxiAjIqKIxiAjIqKIxiAjIqKIxiAjIqKIxiAjIqKI\nFnFBtqnCDQAoqvdiztoabKt0h7kiIiIKp4gLsqV7bFj8nQ2PbbHgR2kavLDTiu0MMyKibiviguyt\nvFhYPAE8PtKIuwbE4HejTfjdTitqXYFwl0ZERGEQcUEWr5Hw7GgTxiSpAQA58Sr8KE2DfxxxAACO\nWX34vMQZzhKJiKgTRVyQteT2/jqsKnGitMGHJ7fX48/f2yHLcrjLIiKiThAVQZaklXBZqhr3bqjD\n6EQVNAoBxTZ/uMsiIqJOoAh3Ae1lVmYM/DJw/2A9PAEZu6o96GuMmrdHREStiIoWGQAk6yQ8NtwI\nlSQgN0GFgmpPuEsiIqJOEDVBdrbhCUp8V+OFn+NkRERRLyqDLF4jIUEjoqjeF+5SiIiog0VlkAHA\nyAQVCqrYvUhEFO2iNsiu7KnBp0ccKLGzVUZEFM2iNsiyY5WYO1CPRVvrYfVw1w8iomgVtUEGANf2\n1mJInBL/KuZOH0RE0SqqgwwAss0KVDm5OJqIKFpFfZDFayTUutm1SEQUraI+yOLUIoOMiCiKdY8g\n4xEvRERRK/qDTBNskXE3fCKi6NQlguyNN97A+PHjMWHCBPzsZz+Dx9N+C5nVkgCVKMDmZZAREUWj\nsAdZeXk5/vznP2P9+vXYvHkzfD4fVqxY0a6vcbpVRkRE0SfsQQYAfr8fDocDPp8PDocDPXr0aNfn\nj+c4GRFR1Ap7kPXo0QMPPvgghgwZgoEDB8JkMmHKlCnt+hpxGhE1bJEREUWlsJ88abFYsGrVKhQW\nFsJoNOKuu+7C8uXLccstt7R4fVFR0QW/hujU4NDJAPo4uYnwxbiYe06Xjvc9PHjfO1dmZuYlP0fY\ng2z9+vXo06cPYmNjAQDXX389tm3b1mqQXcybzkADLB4ZmZn6S6q1OyoqKmqXDxpdGN738OB9j0xh\n71rs2bMnduzYAZfLBVmWsX79emRlZbXra8RpRNS4uE0VEVE0CnuLLDc3FzfccAPy8vKgUCgwdOhQ\n3H333e36GvFqblNFRBStwh5kALBw4UIsXLiww56f0++JiKJX2LsWOwO3qSIiil7dIsiMKgEOnwyP\nn7t7EBFFm24RZKIgwKwWUcfuRSKiqNMtggzgcS5ERNGq2wRZPCd8EBFFpW4TZMlaCWUNXEtGRBRt\nuk2Q9TUqcMzmC3cZRETUzrpNkPUzSDhqZZAREUWbbhNkfY0KFNv8CPCkaCKiqNJtgkyvFGFUCSh3\ncJyMiCiadJsgA4B+BgWOWhlkRETRpFsFWV+jAsc4TkZEFFW6VZD1MypwlDMXiYiiSvcKMkOwRVbj\n8uOBb2tRzTPKiIgiXrcKsl56CeUOP57aXo9jNj++q/GGuyQiIrpE3SrIVJKA1BgJCRoJszJ12FfL\nICMiinRd4mDNzvTMKBOStRKK6r14fZ893OUQEdEl6lYtMgDobVBAoxCQZVbiuM0Hp48LpImIIlm3\nC7LT1JKAvkYFDlrYvUhEFMnaHGQbNmxAcXExAKCiogL33nsvHnjgAZw6daqjautwg2OV2Fd3/iD7\n1zEHdlV7OqEiIiK6UG0Osl/96leQJAkA8OSTT8Ln80EQBMybN6/DiutoQ+KUTSZ8OH0y9tQ0D6yd\n1V4ctHD9GRFRV9TmyR7l5eXo1asXfD4fvv76axQWFkKlUiE7O7sj6+tQg2KVWLzHhoAsQxQErC11\n4W+HGvDJtHgIghC6zuoJwO7loZxERF1Rm1tkBoMBlZWV2LRpE7Kzs6HX6wEAPl/ktlSStBJiVSL2\n1wXfw9ZKDyqdAZT+4ADOek8AVg8nhRARdUVtbpH9/Oc/x9SpU+HxePDiiy8CAPLz85GZmdlhxXWG\ny1LVWF/uwgCzAjurPRidqMKuai966s/cGqtHho0tMiKiLqnNQTZ//nxcd911kCQJffv2BQCkpqbi\n1Vdf7bDiOsNlPTRYtM2CiSlqpOokTE1TY1ulB9f30QIAZFlubJExyIiIuqILmn7fu3dvlJeX45//\n/CcAoEePHujTp09H1NVp+hklKEUB7x9qwNhkFUYmqLCr2hM6gNPhk+GTAZuXXYtERF1Rm4Ns3759\nyM3Nxbx58/DQQw8BADZt2oQHH3yww4rrDIIg4LJUNXZUeTE2SY1knYQYpYhjjeeWWT0yRIBdi0RE\nXVSbg2zBggVYtGgRtm/fDoUi2CM5ceJE5Ofnd1hxnWVKqhrxahHZ5uD7GpmgxM7GdWP1ngBSdCIn\nexARdVFtDrL9+/fj1ltvBYDQ1PSYmBg4nc6OqawTZZqUWDYtHgox+L5y4lT4vnGhdL03gB46CU6/\nDL/MMCMi6mraHGTp6enYvXt3k8cKCgrQr1+/di8qHNTSmXVjPXQiTjmDXYv1bhmxahExCgF2jpMR\nEXU5bZ61+MQTT+AnP/kJ7rnnHni9XixevBh/+ctfIn7WYkuSdRIqncExsXpPAEaVCINShM0TgEnV\nbbenJCLqktr8U/nqq6/G8uXLUVNTg4kTJ+LEiRP44IMPMHXq1Esuor6+HnPmzMGYMWMwbtw47Nix\n45Kf81LEa0TUewLwBmRYvcHwMqoEzlwkIuqCLug8suHDh2P48OHtXsTChQtxxRVX4G9/+xt8Ph8c\nDke7v8aFkAQBcWoRVc4A6t0y+hmDLTIrZy4SEXU5bW6Rvf7669izZw8AYPv27RgyZAiGDRuGbdu2\nXVIBNpsNW7ZswaxZswAACoUCRqPxkp6zPSRrJZxy+s90LaoE2DhzkYioy2lzkP3pT39C7969AQDP\nPPMM7r//fixYsACPP/74JRVQXFyM+Ph43H///cjLy8O8efO6xEzIJK2ISqf/TNeiUuRaMiKiLkiw\nWCxtamb06tULJ06cgM1mQ05ODo4cOQJJkpCeno6SkpKLLmD37t2YNm0avvrqK4wYMQILFy6E0WjE\nokWLWry+qKjool/rQnxWqYZSAHbYlPhpqgMFViWUAnBdortTXp+IqDtoj/162zxGlpaWhq1bt+LA\ngQOYMGECJEmC1WoNnVF2sVJTU5GWloYRI0YAAH784x9j6dKlrV7fWZsUZyudOGjxwm33IKd/H9SU\nuVHp9CMz03DOP+fwBWBxy0iNubT70lUUFRVF/MbQkYj3PTx43yNTm4Ps2WefxZw5c6BUKvH+++8D\nAL788kvk5uZeUgFJSUlIS0vD4cOH0b9/f6xfv75LnHGWrBWxoTw4RmZSiTAoBRypP3/j9eXdNlS7\nAnh9UmwnVElERG0OsiuvvBIHDhxo8tiMGTMwY8aMSy7i97//PX72s5/B6/WiT58+eOONNy75OS9V\nklZCsc0PSQguljaqzj9GtrbUhSP1PlS5AnD75SaLrImIqGO0OcgOHDiAuLg4JCUlwW6349VXX4Uk\nSXjooYegVCovqYicnBysW7fukp6jvSXrRFS7AkjWBufDGJQCrI3ryN47YIckCBibpMLA2OB7t7gD\neK3QhhfGmvHHQhsOWLwYFq8KW/1ERN1Fm2ctzp07F/X19QCAp556Cps3b8b27dsxf/78DisunHSK\nYHeiUXU6yII7e9i8AXx6xAmXT8avtlhwyhHcymrLKTeGJQSDbWi8Et/VeNutlq2VbpQ7/Oe/kIio\nG2pzi6ykpASZmZmQZRn//e9/kZ+fD41Gg2HDhnVkfWGVpJVgUgW7B0/v7LG72oshcUrcO1iPapc/\ndAjnjioPRiUGW2BD41T4d3H7LSH49IgTk1JU+HFfXbs9JxFRtGhzi0ytVsNms6GgoABpaWmIj4+H\nWq2G2x2909GTtWJob0V94zqygioPchOD3YljktTYWulGQJZRcFaQ5cQrsa/OC1+gbQuofQEZhTWe\nVr9f4/Kj0nVmfM7G06qJiELaHGQ333wzbrjhBtx333244447AADfffcd0tPTO6y4cEvWSaGuRbUk\nQBSAzafcyG0MrDFJKuyq9uKQxQeDUkSKLjjl3qQSkaQVcdjqa9Pr7K7x4rmd1la/X+MOhLowAeCu\ndbWodrGrkYgIuICuxRdffBFr166FQqFAXl4eAEAURbzwwgsdVly4jU9WwXdW48egFOH2y8gwBm+b\nWS2il17CXw82hMLttJw4JQprvMg2n38izCGLF9WuAAKyDFFoOtPRG5BR75FR1dgis3kCqHMHUOMK\nIEETHWvViIguxQVtGjx16lScOHEC27ZtQ48ePUKLmKPVmCR1k68NSgF9DMomYTMmSYW/H3LgudGm\nJtdmm9s+4eNQvQ9+GahzBxD/g3CqdQUgAqHz0coaW2YWdi8SEQG4gK7FiooKTJ8+Hbm5uZg9ezZG\njhyJ6dOno7y8vCPr61KMKrFZy2tskhqiAAxPaNry6m1QoMTetq7FQxYvYhQCqpzNw6naFUBfowI1\nrgD8shyavVjvZpAREQEXEGSPPPIIhgwZgmPHjuHgwYMoLi5GTk4OHn744Y6sr0t5cIgeP0rTNHls\nYKwC/zveDL2y6a3spZdQYvdDls894SPYVShjaLwSlc7m41617gBSdCL0CgF17gBKG063yLgTPxER\ncAFdi/n5+Th48GBo8XNMTAyeffZZDBw4sMOK62oyTc3Hu0RBwIiE5gufTSoRKjE4UeNcY1lF9T5k\nmBRI1kqhcbCzVbv8iFdLSNIFUOkMoNzhR6JGhIUtMiIiABfQIjObzc22qCoqKoLJZGrlT1C6XoES\nW7AFFZDlFltnh+q9GGBSIFErtti1WOMKIF4jIkkjodLpR1mDHwNjlRwjIyJq1OYW2bx58zBjxgzM\nnj07dKTLhx9+iCeeeKIj64to6QYJJXYfRiaqsGhbPa5N12Jyj6YTSA5afBiXrIIoCDhibb4mr8YV\nwOA4JaxeEZXOAMocfszoo0ZhbXDdmcsn48sTTi6WJqJuq80tsjlz5uAvf/kLampq8MUXX6CmpgZv\nv/02SktLO7K+iNZbr0CJ3Q+HL7iQesup5kF1qN6HLJMSiZpWWmTuMy2ysgY/alwBDDArQl2LR6w+\n/Hl/Q4e/FyKiruqCpt9fdtlluOyyy0Jfu91u3HzzzWyVtSJdL2FrpRu7q71I0IjYXumBLMsQGqfv\nry9zweoJIN0g4ZRDaHGyR3C9WHD92uqTLsRrRMRrxNBkjwqnHw0+GTZvAAZlm/9fQkQUNS75J9/5\nZuV1Z+mNLbKtlR78uI8WggCU2INh9f8O2PHmPjv+MN4MSRCQoBFR6w4uij5bjcuPOLWIRI2EI1Yf\nUnUSzKozkz1O7/hRwU2FiaibuuQgEwSeudWaJJ2Iek8AmyrcGJOkxqhEFXZUebCzyoMvT7rwVl4c\nBjTu/KGShNAU+9O8ARk2rwyzWkSyLvhXlRojQa8U4PLL8AZkVDiC15/+lYiouzlv1+L69etb/Z7X\n235HlUQjSRDQM0YBqyeAvgYJoxNV+OKEC6tKXLh3kB6x6qb/j0jUSqhyntndo84dQKxahCQIjb8C\nqToJohA8XqbeE8Appx89dCJbZETUbZ03yB566KFzfr9nz57tVkw0StdLiFEqIAgCRiaq8GyBFUPj\nlbjsB7MXASBRK6LS6UdPvQSHT0Z149R7AKHux9SYYMiZVQIs7mCQDYtXoaKF8TUiou7gvEG2Z8+e\nzqgjat3cTwe9Mtj9alKJmJ6uwcy+2ha7ZBM1EipdATyzw4qTDT7cPSAG8We12q7oqQltQmxWByd8\nVDj8uDVDh40VbTtOZ2eVB8MSlJDYJUxEUYLT3DrY4DglehvO/H/h0eFG9G9hhxAASNKK+PSoAzZv\nACMTVHhjrz3UIgOAuQP1oaNizCoRJXYflKKADKOiTWNksizjqe31OFzftj0giYgiwQVNv6eOlaiV\nUOMK4PdjzUjWSthX50ViK9tbmVQiDtb5kKyVkKKT2jRGVueW0eCTUdrgD00yISKKdAyyLmR8sgqv\njDOHWnBLJ8RC0Uqb2awWsbvGhbQYCQalABk471qyEw3BltjJBo6nEVH0YNdiF6JXihh+1gbEZrXY\nbFf90PdUAo7b/EjRSRAEASlasckp0hZ3oNnGwifsfihFhHbQJyKKBgyyCGVWi5ABpGiDXY/B7sUz\nwfXXgw349VYLfIEzC6xP2P0YmaBCqZ1BRkTRg0EWoUyq4F9dsjb4a4pOajIF/4DFC4s7gE+OOEKP\nnbAHNygubeBkDyKKHgyyCGVWnQmw07+envDh8cs4ZvPh5XFm/OOIA8dtweA6YQ+uOXP6gQYvdwIh\noujAIItQ5sb1Zcmng0wrorwxyI5afegZo0BvgwIz++rw72InfAEZFU4/0mIkpMVIHCcjoqjBIItQ\nRpWAGX20MDYuth4Up8SeGi/cfhkHLF4MMAdnPv4oTY1vytwobfAjQSNCLQlIi5E4c5GIogaDLEJJ\ngoD5Qw2hHUISNBIyTQrkn3LjgMUX2gGkl16BeI2I/x53opc+GG492SIjoijCIIsiV/TU4KuTLhyw\neJFtPrNEcGqqGv857kQvfbAbki0yIoomDLIoMrmHGruqvahw+NHPeCbIpqRp4PYDvRo3HO6pZ4uM\niKIHgyyK6JUiRiWq0M+ogEI8sylwD13wCJnMxj0e02IknLD7sL/OG1pELcsyPixqwKKtliYLq4mI\nurous0VVIBDAlClTkJqaio8//jjc5USsn2TocMLefJ3YK+PNod/Hq0XkxCmxtNCGCocf2WYlNJKA\nCqcf45JU+PmGWiwaacTYpOBRM26/DLXE3fKJqGvqMkH2pz/9CdnZ2bBareEuJaINjlNicNy5NwQW\nBAHPjQkGm9sv46uTLpy0+/H4CCM0CgHZsUq8vc+OMYkqBGTg/m/rcPeAGExu4Qw1IqJw6xJdi6Wl\npfjqq68we/bscJfS7aglAdf11uLewXpoFMFW17gkFbwysKfWi0K7AkesPhRUecJcKRFRy7pEkC1a\ntAjPPvtsi4dNUucTBAEz+2rxr2NOfF6jxq0ZOuyuYZARUdcU9q7FL7/8EklJSRg6dCi+/fZbyLJ8\nzuuLioo6qbLurb8feKfCCL0k4DJFOf7TYMTO/YdhUJz774faDz/r4cH73rkyMzMv+TnCHmRbt27F\n559/jtWrV8PlcsFut+MXv/gF3n777Ravb483TW1zh9gAyVaFQQMykVNnQYPZjJEcJ+sURUVF/KyH\nAe97ZAp71+LTTz+NvXv34rvvvsN7772HyZMntxpi1LlmZ8VgtNELABgWr8TuGg9kWcbmCjf852k5\nExF1lrAHGUWG4fEqfFftxev77Hhyez0+KnKc/w8REXWCsHctnm3SpEmYNGlSuMugFgwwK3CywQdB\nAN69LA4LtlgwLF6JnHjV+f8wEVEHYouM2kQhCnhipAmvjDOjn1GBR4cZ8OIu63kn5xARdTQGGbXZ\n5B7q0DloE1KCkz6O2bidFRGFF4OMLlpuoiq0ULrC4cfK485WW2hsuRFRR2GQ0UXLTVShoDoYZB8f\nduC1vXa8tMsGj79paO2s8uCWr2pwoM4bjjKJKMoxyOiiDY9XobDGiwZvAGtLXXg7LxZOv4wXd53Z\nL/OfRx14bqcVmSYF1pe7w1gtEUUrBhldNLNaRGqMhLe/b0B2rBK9DQo8MdKIgxYfNle4savagw+K\nHPjT5FjclRWDzRVtDzJZlvH7XVbs4B6PRHQeXWr6PUWe3AQVPj7iwDOjjACCmxAvGGbA73dbEZCB\nhSMMSNZJSNSKsHlllDb4kBZz/o/dp0edWFPqgloSMCqRU/yJqHVskdElGZWkQqxKCM1iBIJjZxOS\n1bg2XYMxjWeaiYKAcckqbK5ovYVV1uDH/ztgxweHGvBhUQMWjjCisJbjakR0bgwyuiS5CUr8v8vj\noRSbnlwwf6gB92Trmzw2IVmNzada7158Z78dx2x+VLsC+M0oE/J6qFHa4IfdG7jkOp/ZUY9a16U/\nDxF1PQwyuiSCIITWlp1PbqIKBy0+lDuarz2rcwewrdKDR4cbMH+oASMSVFCKAgaYFfi+zgtZlvFh\nUcNFhZrFHcC6MjeKbc1PziaiyMcgo06jVQi4KysGvyuohy8g4z/FTsxeW4NKpx9flDgxuYcaBmXT\nj2ROnBKFtV4UVHvxzv4G/Pe484Jfd1/jtP8KJxdvE0UjBhl1qp9kaGFQivjlpjp8fLgB45NVWJhv\nwf8dd+L63tpm1+fEKbG31ou/H2zALf20WHHUCV/gwhZX7631QiMJONVCS5CIIh+DjDqVKAh4fIQR\nvfQKvDopFvcN0mNovAo6hYhBsc1nMw6KU2JPjRe17gB+MUiPtBgJ68oubD1aYY0Xk1JUqHBwjIwo\nGjHIqNOZ1SIeH2FEgkaCIAiYl6PHa5NiIQhCs2sNShEZRgVmZ+mgEAX8JEOHjw87YPO0LZTcfhmH\nrV5MTdOwa5EoSnEdGYWdIAjQnuOT+MeJsdAqgiE3LlmF/FMezF5bg9lZMZjZVwuxhQD8+qQLKToJ\nfllGb4MCvQ0SKti1SBSVGGTU5Z0OMSDYNfnIMANm9tXiD98Fd/5YNMIIg+pM58LHhxvw2TEn3H4Z\nA2OVyIlTIkkrodYdgC8gQ9G4VOCo1Ye/HWzAM6NNnf6eiKj9sGuRIlJfowJ/nBiLHjoJj+VbQrvr\n/6fYiZUlLrw2KRbPjTFjT40Xw+KDU/lNKhHVZ60lW1/mwvpyN0rsnJZPFMkYZBSxFKKAB4fo4fTL\nKKjywuWT8deDDfhNrhFJWglD4pT4eFo8JqUEt7hK0Uk4ddY42bZKD7JMCqw56QrXWyCidsAgo4gm\nCgJu76/DssMN+G+JE4NiFehvUoa+b1CJoUkkKVoxNE5mcQdQYvdjXo4Ba066IMsyvilzYcVRB89O\nI4owDDKKeD9K06C0wY+/HmzAXVkxrV6XopNwqnEKfkGVB8MTlBgUq4BCFPDegQb8sdCOlced+N89\nNpQ7/Dhq9SHQgaF2zOrDulK2BokuFSd7UMRTiAJmZcZgZ7UHWWZlq9el6CR837jLx9ZKD8YkqiAI\nAqb11OCjIgeWTDQjXS/hpV02zNtUB69fxm39Y3Brf12b6vDLMgprvFBJAgaaFS0uJzjt23I3/vCd\nFX45uPGyQSmisMYDlSRgwDneAxE1xyCjqHB9Hy2u66055zXJWhFrS/0IyDK2V7pxz4Bg6+0n/XSY\nmqpGT33wn8OzjbMYD1m8WLStHjf20zbZFFmWZfzzmBMbyt14fowJeqWItaUuvLHXjli1CHdAhj8g\n4+lRJmS3EErFNh/+9zsrfj/OjE8OO/DlCRdu6K3FczutGJ2kYpARXSAGGUWNc7WAgGCLrLTBj+d3\nWtHPqECPGAkAoFEIoRA7W5ZZiXS9hK9PunB1enD7LG9AxlPb61HvDqCXXoFnC6yY0UeL1wpteHGc\nGdlmJWRZxqdHnfjbwQa8ONbc7Hm3nvIgr4cG2WYlZvbV4pXdNsgyICO4JICILgzHyKjbSNJKOOUM\nwOMHXmghYFpye38dPj5yZgLIquNOePwyXpsUi8eGG+APyHi2oB7PjTGHWl+CIOCGPlrsr/PiRAtT\n+3dUeTA6KTiTMidOCZUk4N0DdiwaYUSxzd9sskm1iwu5ic6FQUbdhloS8OpEM3472gi1dO7W22mj\nElXQSgI+ORJcYP1+kQM/H6SHQhSgEAX8bowJb06Ow+C4pt2BaknAdb2Dmxyfze2XsbfWi+EJZ0Jv\ndpYOP+6WH7MmAAAapElEQVStxfAEFXQKARXOM2vd/LKMe9bVhlpqNS4/5m2qu5TbQBR1GGTUrQyN\nV0E6Txfk2QRBwG9HmfDpUQeeLajHQLOyybiXTiGin7HlHvqZfbVYU+rCN2Wu0Blse2u96GuUmhxX\nMyVVg/uHGAAAfQ0KHDure7Go3gebV0ZRfXCSyv46H76r8aLmElppnsYwJYoWDDKi80jWSXhujAmF\nNV7ck9369P4fitdIeHCwHqtPuPCL9bVYVeLEjioPRiWqWv0z/YwKHDvrANBdVR6oJeBwffCx04F2\nwHLxY2lbKz14apulU9bL/ac42BVL1JEYZERtkG1W4rOrE1ptfbXm6nQtXhhrxhuTY/He/gZ8XuI8\nZ5D1NUhNJnzsrPbiyp4aHLGeDjIf+hgk7K+7+BbVrmoP6jwyTjR07NibLyDj1UJbk2Am6ggMMqI2\nupAuyR/qpVfgpXEmpOsVGBTb+vT6fsYzXYveQLAL8OZ+Ohyu90GWZRTV+3B9by0OWC4+yHZXe5Cu\nl7CnpmO7F8sdfvhk4ISdk1WoYzHIiDpJpkmJVyfFhnbfb0lvgwInG4IBsL/Oi156Cb0NCkiigMNW\nHxw+GZenanDA0vKuIy6ffM4TtC3uAE45A7iprxbf1Xja5X21pqQxwFqauUnUnsIeZKWlpbj++usx\nduxYTJgwAW+99Va4SyIKG7UkIFkn4ZRbxK5qL0YkBLsh+xsV+LzEhf4mBeI0ImIUAkp/0DV40u7D\n3d/U4JMjjlaf/7saD3LilBiRqGpzi+xit+kqsfkQqxY7vAuTKOxBplAo8Pzzz2Pr1q1YvXo13n33\nXRw6dCjcZRGFTX+jAktKYvDR4QaMS24MssZd+jNNwTG6gbFK7K8709I5avVh3iYLRieqsLbU3epz\n764JTv3vFSPB45fPe9hojcuPW1bX4Ej9hbeqSux+TEhWsWuROlzYgyw5ORlDhw4FAOj1emRlZaG8\nvDzMVRGFz4JhBjzWpwEfT0s40yIzKWD1yqEgyzYrcbBxnMzbuCh77sAYzB9qQJ070Kw7zxuQEZBl\n7K72YHhCcI/JofEq7DlP9+J7BxqgVQQXbANAWYMf/zhHi+9sJXYfJqaoccLefJE3UXsKe5Cd7fjx\n4ygsLERubm64SyEKG71SRJIqALP6zD/P/o2zJTMbj6gZFq/EujI3Dlq8+OiwAyk6CVf30kASBOT1\nUOObsjOtsmqXHzd+WY1p/61CjTsQeq5h8UrsPkf3YlG9F1tOefDapFgctfqwucKNX2+14C8HGs7b\n3SjLMo7b/RgUq4RWQpMDTYnaW5cJMrvdjjlz5uCll16CXq8PdzlEXUpPvYTLU9VI1wf3hxwYq8Qv\nc/R4LN+CFUcdeDjHENpr8vK0pkH214MNuDZdi9XXJmL5FQmhySYTU9TYWO6Gy9c0lA5ZvPj7oQY8\nu8OKu7N0iFWLmDMgBk9sq8f4ZBVMKgFl5xn3qnPLEAXArBbRS684Z/diUb0Xh+u5QJsunmCxWMLe\n5vf5fLj11lsxbdo03Hfffee8tqioqJOqIur6TrpEWH0iBunPdCUGZOCxIgP+J80Bs0LGy8dj8FyG\nDY17JDfx2gkdRhq8mGj24rhTxKeVWlR6RIwyejFA58MQvQ+iAPhlYGu9EuNMXrx5UofxJg9yja2P\nmx1skPCvKg1+3acBfyvTorfWjymxzbsxAzLwzDE9eqj8uLens4VnomiXmZl5yc/RJXa/f+CBBzBg\nwIDzhhjQPm+a2q6oqIj3PAzaet9bu2KB3oU39ing8suYNUCH4f1TWrzuNoMb7x9qwPQhJry5oQ53\nD4jBNemaJsfWnJbd+OvQgB0OAJmZzXtOCms8SNJKCFR6kAUvMjNTMURoQLUrgMxMQ7PrV59wQaVq\nQJErgH790y5prV574Oc9MoW9azE/Px/Lly/Hhg0bMHnyZOTl5WHNmjXhLosoouWlavDBj+Lxq2FG\n3Ni39YNBxyarUOMOYMEWC27qp8UNfbQthtjZMoyK0E4jZztS78OibfX45aY6bKt0I73xaJzWuhZ9\nARl/PWjHvBwDYtUiii5gZuSuag+Wt3HSCUW/sLfIxo0bh9ra2nCXQRR1lKKAyT3U57xGEgTc1FeH\n7+u8uL2NJ2FnGBV46wdBZvMG8NT2eszLMcDhk7F4jw3TG89w6xkjNQkyqyeAz445seWUG2kxCoxI\nUGFUogoFVZ4WDyL9oRqXH78rsMITkDEuWYVeLZwldzEuZL3c3lovrJ4AJqSc+/5S5wh7kBFReP0k\nQwtBaFuIAUBqjASLW4bdG4BGEvDFCRc+LGrA5BQ1pvUMntLdxyCFQik1RoInIGNZUQOu7KXBo1ss\nyDIpMTdbj5z44DW5iSosP+rAnZnn3pQ5IMt4fqcV1/XWQCMJeGd/Q+hE77O5/TIUYtNtxY5ZfSi2\n+XB5WssniS/aWo+hSmWr3bWnlTv8WLTNgvQYBYOsi2CQEXVz5ztZ+4ckQQhtbvzlCReO2/14fIQR\nQ+PPbIZ89u+VooA3J8fiN9vr8fdDDszK1OHOTF2T1x2eoMTvCnxw+WRoFK3Xk3/KA5tXxl1ZMfDL\nwKyva7C31oshZ50HV9rgwwPf1sHtB/oZJfTQSfDJwT0mvQFgTJIKMcqmoyrFNh/yKz1w6ZW4o/Ex\nWZab3RuXT8ZT2+pxSz8dPixywO2Xz3u2nSzLOGr1I8PU8T9uF2614N5BevQxdK8f7WEfIyOiyJNh\nUuCTIw7sqPLg92NNTYKrJUlaCX+cGIuXx5kwKyumWUDoFCIyTYrz7v+4scKNK3tqoBAFqCUBcwfG\n4LW9NvgbuwUdvgCe2FaPuwfE4B9XxONnA/UYnaTCiHglPvxRPIbFK7Gpovlr/OuYEzP6aHHAoYDL\nJ8PhC+CudbU49IPNmVedcCJJK2JWpg59DVKbNm/eb/HhZxtqQ2fSdRS3X8b2Sg/yT7W+s0u0YpAR\n0QXrZ1RgU4UHjw43NmvdtEYlCecMvCmpanx+wtXq9/2yjC0Vbkw8qzvvip7BGZYrj7vg8AXwm+1W\nDIlT4sd9tDCoRAxPUOGqXlr8uK8OMUoRl6dpsLas6Ws4fAF8XerCHZk69Nb4UVDtwX+Pu+DyyVhS\naAuNncmyjJXHXbipX7A1mROnbNN+lftqvVAIwCeHO3ZyyjGbD7IM7Kjq2M2guyIGGRFdsInJaszP\n0Z/zbLULdVUvDQqqPKh0ttxy2V/ng1ktIvWsBXGiIODhHAP+csCOeZssSNSKmH/W4vBmdaeoUFgT\nnKhx2srjLgyPVyFJK2G4wYt1pS7844gDz40xQQSwqiQYfAfrfXD4AhiREOzGzIlv28bL39d58dPs\nGHxd6rqkk73Pp8gS3BJsX60P7i50mOnhei9e2GkNfe2Xz31Cw8VgkBHRBUvWSZhxjmn9FyNGKeKK\nnhr8p7jlhdGbftAaOy3DpMAtGTr8KE2NR4cZznlMjk4hYlSiChvKg91v2yvdWFbUgJ82nvw9XO/F\nmlI3+hoUGGBW4uGhBryz346CKg9WHXfhmnQtxMaQzIlT4vs6b6hbszX76ryY1DgR5tOjHbfou6je\nhxEJSvQzSthb23rAunxykyDvaF+XurGuzBUK1w8POfDGPnu7vgaDjIi6jJl9tVh53Bn6oSfLMrZX\nurGv1ouNFW5MamWW4J2ZMbitf/Oxt5ZM763Bm/vsWLjVgud2WvHMaBP6Nu4/maCSMS5JhbuygiHd\n36TEb3JNeH6nFatPunB1rzMzHs1qEfEascmJ3ketTVtDVU4/3H4ZaTESbsvQYeVxZyhEjtt8oUNU\n20qW5WZbip1WVO9Ff5MCuY1LGVri8AXw8JY6LNpaf0EbORfVe/HSLuv5L2yh3m/L3dBIQmi8cUul\nG1+fdLVrq4xBRkRdRi+9AjnxKryy2wpfQMZ7BxqwZI8Nr+61IUYhIMt86bPxxiap8dG0eFzZU4MX\nx5qbjdu9NM6MnLMeG5mowlt5sZifo0eStuk+X8PilVh53AW/LGNzhRv3bqjFwq0WOBvDZl+dF4Ni\nlRCE4DlzE1PU+OxYMKif2FaPP31/YS2TlSUuPLHN0uxxX0DGMZsP/U2K4Jq86uZB5vbLWLS1HhlG\nBey+ALacavtY2rZKD7444ULRBe6Jedzuhycg48qeGhTWemH3BlBs9SNFJ7UatheDQUZEXcoTI41o\n8Mm455tabKxw483JcXg7Lw5v5cWFuvUulUklYmqaBoNiz78AGwjOury6cYH32e7KikGxzYf7v63D\nK7utWDoxFilaCb/Ot8DmDeD7Wi8Gn/Uat2fq8NkxB97bb0fPGAn7ar2ocwdbaO/st+Pt7+04avW1\n2lpaV+rCrhov6htbdX5ZhizLKLH7kaiVoFOIGBirxEm7Hxb3me5Db0DG09vrkaAR8chQA/4nW4/3\n2nCKwWnf13mRZVKEukbbOs61sTzYis6JV6Kw1os9NV4MjFXgyl4arC1rv9mVDDIi6lLUkoDfjTbh\nml4a/O94c5PjbLqaRK2ExRPMmJ6uxYtjzRgUq8Sjww3INCtw/7d1yK/0YPBZa9zS9cGdTP5b4sKC\nYQaMT1ZhXZkLhyxefNE4qWThVgtu/7oGS/fYmoRRvSeA/RYfchNUoSn2L+2yYtG2ehTWBoMGCK7b\nG5WowpbGa/yyjN8VWKESBSwcYYQoCJiUooJKAh7aWIf/+aYWmyuahorDF8Ajm+vg9geDcn+dD48O\nN2BThRt7ajyY+00t/n6o4bz359vG7uCcOCX21nqxo8qDkQkqTElVY3OFu90mpXTdTwgRdVsKUcAd\nmTGI17SwZX8XIwoCftxHi+zGlpcoCHhoiAG3ZuhgcQeabbt17yA9nh9jQqJWwhU9NVhz0oX3DjRg\nVpYOvxikxyfT4vHCGDMkEfjFhlrsa5y4sbnCjdwEFX6UpsamCg9O2n3YVumBRhLw+l5b6Kw6AJjU\nQ42NjeG0vsyNSqcfT+UaQxNhBCH4n4VZWTG4rb8Ofyy0NQmVzRUe7Kz2YnulB5XOAGQEz8SbmqrB\ngi0W9DcpUHiOCSUAUOHwo9zhx9B4JeI1EowqEV+ecGFkogoJGgn9jApsbac1bwwyIqIOcF1vLf51\ndQK0P9ipJFknhU7+zk1UoazBj+N2H67rHey6FAQB/YwKPDTEgHk5BjyxzYJNFW58W+5GXqoa45LV\nKKjy4G+HGjCjjxZP5hoxKzMGE1POjOuNS1JhV7UXTp+Mfx514s7MGKh+sANJgkbC+GQ1ruipQZZJ\niU+Pnlnn9k2ZC4NiFVhf7sL3dV4MilVAEAT8z8AYvJ0XhwcGG3DI4jtn1+Tqky5cnqoJhWdOY8v0\ndMvxpr5a/P1Q+6ytY5AREXWQ843pnW55PjjY0OKpAxNS1HhpnBl/2G3FzmovxiWrYFaL6G9S4Nty\nN27sp4MkCJgzIKbJ5skGlYiBZgU+LGpAlcuP8cnnXu937+AYfHLEgUqnHw5fADurvfj1cCO2nPLg\nuxpvaCzRpBLRz6iAWS3CqBKanWpQVO8Njdt9ecKFq86a5TkyQYncRFUo2Cb3UEPZTg3u7rUhFxFR\nF/OTjHOvx8s2K/HKeDMKqrwwNO6icl1vLUYnqmBStd4WmdRDjVcL7fjFoJhzrq0DgLQYBW7N0OE3\n2+txQx8tcuKU6G1QoJ9Bgc9POPHiGHOLdR20eNG7cV/Hjw878Pb3dtyVpcOoRBVEARh41izTK3pq\nMPWsDZsFQcAvBjY/0+5isEVGRNTF9TcpcetZx+xc0VODWVnnPilgYooacWoxdJzO+dzRX4cUnYTF\ne2yYkhpcr3dZqhoePzCghWUPA8xK7LcE18H97WADPi9x4k95sVhZ4sJre+24upemybo+QRCaBerw\nhPbZGYZBRkQUhZK0Ev5xZTyM52i1nU0QBDw23IiremlCC8+npKpxXW9Ni/tpZpsVOGDx4oTdhxXH\nHFg6MRbZZiWezjWixO7HFT1bPi6nI7BrkYgoSkkXuO5OqxDwq2HG0NfxGgkLzvr6bFlmBY5ZfXj7\neztuzdAhtnGZxNB4Ff59dcJ5j7dpT2yRERHRBdMpRKToJOyv8+GmH+y72ZkhBrBFRkREF2lKqga9\nYqRzHobaGRhkRER0Ue4ecO4JJ52FXYtERBTRGGRERBTRGGRERBTRGGRERBTRGGRERBTRGGRERBTR\nGGRERBTRGGRERBTRGGRERBTRGGRERBTRGGRERBTRukSQrVmzBqNHj0Zubi6WLl0a7nKIiCiChD3I\nAoEAHn30UaxYsQL5+fn49NNPcejQoXCXRUREESLsQVZQUICMjAykp6dDqVTipptuwqpVq8JdFhER\nRYiwB1lZWRnS0tJCX6empqKsrCyMFRERUSQJe5C1RLjA47mp42RmZoa7hG6J9z08eN8jU9iDLDU1\nFSdPngx9XVZWhh49eoSxIiIiiiRhD7KRI0fi6NGjKCkpgcfjwYoVK3DNNdeEuywiIooQinAXIEkS\nXnnlFdx4440IBAKYPXs2BgwYEO6yiIgoQggWi0UOdxFEREQXK+xdi+fDxdKdJycnBxMnTsTkyZMx\ndepUAIDFYsHMmTMxatQo3Hjjjaivrw9zldHhwQcfRGZmJiZMmBB67Fz3+rHHHsPIkSMxadIk7Nmz\nJxwlR7yW7vlLL72EQYMGIS8vD3l5eVizZk3oe4sXL8bIkSMxZswYrF27NhwlR4XS0lJcf/31GDt2\nLCZMmIC33noLQPt+3rt0kHGxdOcSRRErV67Et99+G/qHu2TJEkyZMgU7duxAXl4elixZEuYqo8Od\nd96JFStWNHmstXv91Vdfobi4GDt37sTSpUvxyCOPhKPkiNfSPQeA+++/Hxs2bMCGDRswbdo0AMDB\ngwfx2WefYdu2bVi+fDkWLFgAWWbn1cVQKBR4/vnnsXXrVqxevRrvvvsuDh061K6f9y4dZFws3blk\nWUYgEGjy2KpVq3D77bcDAG6//XasXLkyHKVFnfHjx8NsNjd57If3+vRnfdWqVbjtttsAAKNGjYLV\nakVlZWXnFhwFWrrnAFoMqFWrVuGmm26CQqFA7969kZGRgYKCgs4oM+okJydj6NChAAC9Xo+srCyU\nlZW16+e9SwcZF0t3LkEQcOONN+Lyyy/H3//+dwBAZWUlkpKSAAQ/kNXV1eEsMapVVVU1uddVVVUA\nmv876NGjB/8dtKN3330XkyZNwkMPPRTq3uI97xjHjx9HYWEhRo0a1exny6V83rt0kLWEi6U7zurV\nq/HNN99g+fLleOedd7B582be7y6Kfy/tY+7cudi9ezc2btyI5ORkPPnkk61ey3t+aex2O+bMmYOX\nXnoJer3+gu7n+a7t0kHGxdKdKzk5GQCQkJCAa6+9FgUFBUhKSgo160+dOoXExMRwlhjVWrvXqamp\nKC0tDV3HfwftJyEhIfRD8q677gp1H/Kety+fz4c5c+bg1ltvxbXXXgugfT/vXTrIuFi68zgcDtjt\ndgBAQ0MD1q1bh8GDB+Oaa67BsmXLAAAfffQRpk+fHs4yo8oPx2bOvtfLli0L3etrrrkGH3/8MQBg\n+/btMJlMoS4ZujA/vOenTp0K/f7//u//MGjQIADBe75ixQp4PB4UFxfj6NGjyM3N7dRao8kDDzyA\nAQMG4L777gs91p6f9y6/jmzNmjVYuHBhaLH0ww8/HO6SolJxcTFmzZoFQRDg9/txyy234OGHH0Zd\nXR3uvvtulJaWomfPnvjrX//a4oA5XZi5c+di48aNqK2tRVJSEhYuXIjrrrsOc+bMafFeP/roo1iz\nZg10Oh3eeOMNDB8+PMzvIPK0dM+//fZbFBYWQhRFpKenY+nSpaEfmosXL8b7778PpVKJl156KbQk\nhS5Mfn4+pk+fjkGDBkEQBAiCgKeffhq5ubmt/my50M97lw8yIiKic+nSXYtERETnwyAjIqKIxiAj\nIqKIxiAjIqKIxiAjIqKIxiAjIqKIxiAjIqKIxiAj6iBbtmzBVVddhfT0dPTr1w/XXHMNdu/ejWXL\nlnGHGqJ2pAh3AUTRyGaz4bbbbsPSpUsxY8YMeDwebNmyBSqVKtylEUUdtsiIOsCRI0cgCAJmzpwJ\nQRCgVqsxZcoUKBQKPPLII9i+fTt69uyJPn36AAA8Hg+efPJJDBkyBAMGDMCCBQvgdrsBABs3bsTg\nwYOxePFiZGRkYNiwYVi+fHnotVavXo1x48ahV69eGDx4MF5//fVwvGWisGGQEXWAjIwMSJKE++67\nD2vWrIHFYgEAZGVlYfHixRg9ejROnjyJ4uJiAMDTTz+No0ePYtOmTdi5cyfKysrw8ssvh57v1KlT\nqKurw4EDB/Dmm29i/vz5OHLkCADgl7/8Jf74xz/ixIkT2Lx5M/Ly8jr9/RKFE4OMqAMYDAZ88cUX\nEEUR8+fPR//+/XHHHXeEDg/8offffx8vvPACTCYTYmJi8PDDD+PTTz8NfV8QBDzxxBNQKpWYOHEi\nrrzySnz22WcAAKVSiQMHDsBms8FkMoVO4yXqLhhkRB0kMzMTb7zxBvbu3Yv8/HyUl5fj8ccfb3Zd\ndXU1HA4HpkyZgj59+qBPnz64+eabUVdXF7rGbDZDo9GEvu7VqxcqKioABEPwyy+/RE5ODq677jps\n3769498cURfCICPqBKdbZPv372922m18fDx0Oh3y8/NRXFyM4uJilJSUoKSkJHSNxWKB0+kMfX3y\n5EmkpKQAAIYPH45ly5bhyJEjmD59Ou65557OeVNEXQSDjKgDFBUV4fXXX0dZWRmAYPCsWLECY8aM\nQVJSEsrKyuD1egEEuw3vuusuPP7446iurgYQPBV37dq1oeeTZRkvvvgivF4vNm/ejNWrV2PmzJnw\ner1Yvnw5rFYrJEmCXq+HJEmd/4aJwojT74k6gF6vR0FBAd58801YrVaYTCZcffXVeOaZZ6BWq5Gd\nnY2srCxIkoTDhw/jt7/9LV5++WVMmzYNtbW1SE1NxU9/+tPQYY4pKSkwm83Izs6GTqfDkiVLkJGR\nAa/Xi08++QSPPfYY/H4/MjMz8c4774T53RN1Lh6sSdTFbdy4Effeey/27t0b7lKIuiR2LRIRUURj\nkBERUURj1yIREUU0tsiIiCiiMciIiCiiMciIiCiiMciIiCiiMciIiCiiMciIiCii/X8jVUEO9hoH\nrgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f4548efaad0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# plot losses\n",
    "\n",
    "with plt.style.context('fivethirtyeight'):\n",
    "    plt.plot(losses, linewidth = 1)\n",
    "    plt.xlabel('Steps')\n",
    "    plt.ylabel('Losses')\n",
    "    plt.ylim((0, 12))\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from checkpoints/-999\n",
      "1.\n",
      "--------------------------------\n",
      "What' s your name\n",
      "Wie ist das Name \n",
      "--------------------------------\n",
      "2.\n",
      "--------------------------------\n",
      "My name is\n",
      "Mein Name ist \n",
      "--------------------------------\n",
      "3.\n",
      "--------------------------------\n",
      "What are you doing\n",
      "Was machst du da \n",
      "--------------------------------\n",
      "4.\n",
      "--------------------------------\n",
      "I am reading a book\n",
      "Ich bin ein Buch Buch \n",
      "--------------------------------\n",
      "5.\n",
      "--------------------------------\n",
      "How are you\n",
      "Wie sind Sie \n",
      "--------------------------------\n",
      "6.\n",
      "--------------------------------\n",
      "I am good\n",
      "Ich bin schön \n",
      "--------------------------------\n",
      "7.\n",
      "--------------------------------\n",
      "Do you speak English\n",
      "Bist du das \n",
      "--------------------------------\n",
      "8.\n",
      "--------------------------------\n",
      "What time is it\n",
      "Was ist das was \n",
      "--------------------------------\n",
      "9.\n",
      "--------------------------------\n",
      "Hi\n",
      "Hallo \n",
      "--------------------------------\n",
      "10.\n",
      "--------------------------------\n",
      "Goodbye\n",
      "Wiedersehen \n",
      "--------------------------------\n",
      "11.\n",
      "--------------------------------\n",
      "Yes\n",
      "Ja \n",
      "--------------------------------\n",
      "12.\n",
      "--------------------------------\n",
      "No\n",
      "Nein \n",
      "--------------------------------\n"
     ]
    }
   ],
   "source": [
    "# let's test the model\n",
    "\n",
    "with tf.Graph().as_default():\n",
    "    \n",
    "    # placeholders\n",
    "    encoder_inputs = [tf.placeholder(dtype = tf.int32, shape = [None], name = 'encoder{}'.format(i)) for i in range(input_seq_len)]\n",
    "    decoder_inputs = [tf.placeholder(dtype = tf.int32, shape = [None], name = 'decoder{}'.format(i)) for i in range(output_seq_len)]\n",
    "\n",
    "    # output projection\n",
    "    size = 512\n",
    "    w_t = tf.get_variable('proj_w', [de_vocab_size, size], tf.float32)\n",
    "    b = tf.get_variable('proj_b', [de_vocab_size], tf.float32)\n",
    "    w = tf.transpose(w_t)\n",
    "    output_projection = (w, b)\n",
    "    \n",
    "    # change the model so that output at time t can be fed as input at time t+1\n",
    "    outputs, states = tf.contrib.legacy_seq2seq.embedding_attention_seq2seq(\n",
    "                                                encoder_inputs,\n",
    "                                                decoder_inputs,\n",
    "                                                tf.contrib.rnn.BasicLSTMCell(size),\n",
    "                                                num_encoder_symbols = en_vocab_size,\n",
    "                                                num_decoder_symbols = de_vocab_size,\n",
    "                                                embedding_size = 100,\n",
    "                                                feed_previous = True, # <-----this is changed----->\n",
    "                                                output_projection = output_projection,\n",
    "                                                dtype = tf.float32)\n",
    "    \n",
    "    # ops for projecting outputs\n",
    "    outputs_proj = [tf.matmul(outputs[i], output_projection[0]) + output_projection[1] for i in range(output_seq_len)]\n",
    "\n",
    "    # let's translate these sentences     \n",
    "    en_sentences = [\"What' s your name\", 'My name is', 'What are you doing', 'I am reading a book',\\\n",
    "                    'How are you', 'I am good', 'Do you speak English', 'What time is it', 'Hi', 'Goodbye', 'Yes', 'No']\n",
    "    en_sentences_encoded = [[en_word2idx.get(word, 0) for word in en_sentence.split()] for en_sentence in en_sentences]\n",
    "    \n",
    "    # padding to fit encoder input\n",
    "    for i in range(len(en_sentences_encoded)):\n",
    "        en_sentences_encoded[i] += (15 - len(en_sentences_encoded[i])) * [en_word2idx['<pad>']]\n",
    "    \n",
    "    # restore all variables - use the last checkpoint saved\n",
    "    saver = tf.train.Saver()\n",
    "    path = tf.train.latest_checkpoint('checkpoints')\n",
    "    \n",
    "    with tf.Session() as sess:\n",
    "        # restore\n",
    "        saver.restore(sess, path)\n",
    "        \n",
    "        # feed data into placeholders\n",
    "        feed = {}\n",
    "        for i in range(input_seq_len):\n",
    "            feed[encoder_inputs[i].name] = np.array([en_sentences_encoded[j][i] for j in range(len(en_sentences_encoded))], dtype = np.int32)\n",
    "            \n",
    "        feed[decoder_inputs[0].name] = np.array([de_word2idx['<go>']] * len(en_sentences_encoded), dtype = np.int32)\n",
    "        \n",
    "        # translate\n",
    "        output_sequences = sess.run(outputs_proj, feed_dict = feed)\n",
    "        \n",
    "        # decode seq.\n",
    "        for i in range(len(en_sentences_encoded)):\n",
    "            print '{}.\\n--------------------------------'.format(i+1)\n",
    "            ouput_seq = [output_sequences[j][i] for j in range(output_seq_len)]\n",
    "            #decode output sequence\n",
    "            words = decode_output(ouput_seq)\n",
    "        \n",
    "            print en_sentences[i]\n",
    "            for i in range(len(words)):\n",
    "                if words[i] not in ['<eos>', '<pad>', '<go>']:\n",
    "                    print words[i],\n",
    "            \n",
    "            print '\\n--------------------------------'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# This model can be improved by using more training steps, better dataset or even with better selection of hyperparameters "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
